package com.bdth.bdthiss.jsoup;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashMap;
import java.util.Map;

/**
 * 爬虫工具类
 * Created by Administrator on 2020/4/3.
 */
public class JsoupTools {
    public static Logger logger = LoggerFactory.getLogger(JsoupTools.class);

    /**
     * 根据URL获取网页内容
     * @param url
     * @return
     */
    public static  Document getDocument(String url){
        Document document = null;
        //获取网页数据，超时时间为5秒
        Connection connect = Jsoup.connect(url).timeout(5000).maxBodySize(0);
        Map<String, String> header = new HashMap<String, String>();
        header.put("Accept-Encoding", "gzip, deflate");
        header.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36");
        header.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        header.put("Accept-Language", "zh-cn,zh;q=0.5");
        header.put("Connection", "keep-alive");
        header.put("Cache-Control", "max-age=0");
        Connection data = connect.headers(header);

        try {
            document = data.get();
        } catch (IOException e) {
            logger.error(e.getMessage());
            return null;
        }
        return document;
    }

    /**
     * 根据规则获取需要分析的节点
     * @param document
     * @param key
     * @param value
     * @return
     */
    public static Element getElement(Document document,String key,String value){
        Element detail = null;
        if(key.equals("id")) {
            detail = document.getElementById(value);
        }else if(key.equals("css")) {
            Elements details = document.getElementsByClass(value);
            detail = details.get(1);
        }else{
            Elements details = document.select(value);
            detail = details.get(1);
        }
        return detail;
    }

}
